package test2.spider;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import jodd.http.HttpRequest;
import jodd.http.HttpResponse;

public class BaseProcessor {
	
	protected String fetch(String url, String encoding) {
		try {
			System.out.println("发送http请求到--->" + url);
			HttpResponse response = HttpRequest.get(url).timeout(10000).send();
			String html = response.charset(encoding).bodyText();
			return html;
		} catch (Exception e) {
			System.out.println(e.getMessage());
			return null;
		}
	}
	
	protected String fetch(String url) {
		return fetch(url, "utf-8");
	}
	
	protected void parse(Document doc) {
		
	}
	
	public void process(String url, String encoding) {
		try {
			String html = fetch(url, encoding);
			if(html!=null) {
				Document doc = Jsoup.parse(html);
				parse(doc);
			}
		} catch (Exception e) {
			System.out.println(e.getMessage());
		}
	}
	
	public void process(String url) {
		try {
			String html = fetch(url);
			if(html!=null) {
				Document doc = Jsoup.parse(html);
				parse(doc);
			}
		} catch (Exception e) {
			System.out.println(e.getMessage());
		}
	}
	

}
